/**
 * Application Name: TCSS422_WebCrawler
 * Group Name: The Other Guys
 * Members: Scott Freeman, Anthony Melcher, Jason Green
 * Date: November 10, 2011
 * 
 * Related libraries: Jericho HTML Page Parser (author unknown) 
 * 					  http://jericho.htmlparser.net/docs/index.html
 */
package jericho_parser_wrapper;

import java.net.URI;
import java.util.List;
import java.util.Scanner;
import java.util.concurrent.TimeUnit;
import model.Page;
import net.htmlparser.jericho.Element;
import net.htmlparser.jericho.HTMLElementName;
import net.htmlparser.jericho.Source;
import controller.Spider;

/**
 * This class will parse a web page looking for URLs
 * and searching the text for keywords. 
 */
public class PageParser implements Runnable {
	/**
	 * The content to parse.
	 */
	private Page page;
	
	/**
	 * The Jericho HTML Parser.
	 */
	private Source src;
	
	/**
	 * A normal buffer to store string values.
	 */
	private StringBuffer strbff;

	/**
	 * Starts the process of parsing a page for URLs
	 * and text.
	 */
	@Override
	public void run() {
		try {
			page = Spider.pageBuffer.poll(1, TimeUnit.NANOSECONDS);
			if (page == null) {
				return;
			}
			
			src = new Source(page.getMy_content());
			processURLs();
			processText();
			src = null;
			strbff = null;
			Spider.DATA_PAGE_LIST.offer(page, 1,TimeUnit.NANOSECONDS);
			page = null;
			src.clearCache();
		} catch (final InterruptedException e) {
			// do nothing
		} catch (final Exception e) {
			// do nothing
		}
	}

	/**
	 * This method finds all the URLs contained within a given page. It is only
	 * concerned with the links that are contained within the HTML Body tag.
	 * 
	 * @throws Exception
	 */
	private void processURLs() throws Exception {
		URI url;
		String ext = null;
		int index = -1;
		List<Element> elementList = src.getAllElements(HTMLElementName.A);
		src.clearCache();
		String path = null;
		int urlCount = 0;

		for (Element element : elementList) {
			url = null;
			try {
				if (!element.getAttributeValue("href").isEmpty()) {
					path = element.getAttributeValue("href");

					if (path.toLowerCase().startsWith("mailto")
							|| path.toLowerCase().startsWith("javascript")) {
						continue;
					}

					if (path.toLowerCase().startsWith("http")) {
						url = new URI(path);
					} else {
						url = page.getMy_url().resolve(path);
					}

					String strURL = url.toString();
					if (strURL.contains("#")) {
						strURL = cutURL(strURL, strURL.indexOf("#"));
						url = URI.create(strURL);
					}
					
					if (strURL.contains("?")) {
						strURL = cutURL(strURL, strURL.indexOf("?"));
						url = URI.create(strURL);
					}

					index = url.toURL().getFile().lastIndexOf(".");
					if (index >= 0) {
						ext = url.toURL().getFile().substring(index);
						if (!Spider.VALID_PAGE_TYPE.contains(ext)) {
							continue;
						}
					}
					
					if (url.toString().toLowerCase().equals(Spider.DONT_GO_HERE)
							|| url == null) {
						continue;
					}
					
					urlCount++;
					if (Spider.sycSetAdd(url)) {
						try {		
						
							if (Spider.urlBuffer.remainingCapacity() != 0 ) {
								Spider.urlBuffer.offer(url, 1,TimeUnit.NANOSECONDS);
							} else {								
								continue;
							}
						} catch (InterruptedException e) {
							// do nothing
						}
					}
				}

			} catch (final Exception e) {
				continue;
			}			
		}
		page.setUrlCount(urlCount);
		src.clearCache();
		ext = null;
		elementList = null;
		path = null;
		url = null;
	}

	/**
	 * This method removes any HTML tags and punctuation from the text. For each
	 * new word found, the word is placed into a collection with an integer
	 * counter. If the word exists already, the counter is incremented.
	 */
	private void processText() {
		int count = 0;
		Scanner scan = new Scanner(src.getTextExtractor().toString()
				.replaceAll("[^\\p{Alpha}]", " ").toLowerCase());
		scan.useDelimiter(" ");
		String item = null;
		while (scan.hasNext()) {
			item = scan.next().toString().trim();
			if (!item.isEmpty()) {
				if (Spider.keywordsMap.containsKey(item)) {
					Spider.keywordsMap.get(item).incrementAndGet();
				}
				count++;
			}
		}
		page.setWordCount(count);
	}

	/**
	 * Deletes everything from a specific point in a string to the end.
	 * 
	 * @param str
	 *            The string that is going to be modified.
	 * @param startCut
	 *            The position in the string to start the deletion.
	 * @return A shortened version of the original string.
	 */
	private String cutURL(String str, int startCut) {
		strbff = new StringBuffer(str);
		strbff.delete(startCut, str.length());
		return strbff.toString();
	}
}
